import requests
import re
import pymysql
import hashlib
import datetime


class GLY(object):
    def __init__(self):
        self.url = 'http://down.foodmate.net/special/standard/8.html'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36'
        }
        self.host = '127.0.0.1'
        self.db = 'app_mark'
        self.user = 'root'
        self.passwd = '123456'
        self.charset = 'utf8mb4'

    def get_url(self):
        response = requests.get(self.url, headers=self.headers)
        response.encoding = response.apparent_encoding
        html = response.text
        urls = re.findall('<A title=.*?href="(.*?)"', html)
        # 去重
        urls = set(urls)
        for url in urls:
            hkey = hashlib.md5(url.encode(encoding='utf-8')).hexdigest()
            tag = '0'
            channel = '食品添加剂标准'
            sitename = '食品伙伴网'
            lasttime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            list_data = [url, hkey, tag, channel, sitename, lasttime]
            self.save_url(list_data)
        print(len(urls))

    def save_url(self, list_data):
        con = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset=self.charset)
        cur = con.cursor()
        sql = 'insert into gly(link, hkey, tag, channel, sitename, lasttime) values (%s, %s, %s, %s, %s, %s)'
        try:
            cur.execute(sql, list_data)
            print('insert success')
        except Exception as e:
            con.rollback()
            print('error~', e)
        else:
            con.commit()
        cur.close()
        con.close()
        


if __name__ == '__main__':
    gly = GLY()
    urls = gly.get_url()